{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data Load"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'1.0.0'"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import re\n",
    "import time\n",
    "tf.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Make sure the vocabulary.txt file and the encoded datasets for Question and Answer are present in the same folder\n",
    "# reading vocabulary\n",
    "vocab_lines = open('vocabulary.txt', encoding='utf-8', errors='ignore').read().split('\\n')\n",
    "# reading questions\n",
    "question_lines = open('InsuranceQAquestionanslabelraw.encoded', encoding='utf-8', errors='ignore').read().split('\\n')\n",
    "# reading answers\n",
    "answer_lines = open('InsuranceQAlabel2answerraw.encoded', encoding='utf-8', errors='ignore').read().split('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# The print command shows the token value associated with each of the words in the 3 datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " -- Vocabulary -- \n",
      "['idx_17904\\trating/result', 'idx_14300\\tconsidered,']\n"
     ]
    }
   ],
   "source": [
    "print(\" -- Vocabulary -- \")\n",
    "print(vocab_lines[:2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " -- Questions -- \n",
      "['medicare-insurance\\tidx_1285 idx_1010 idx_467 idx_47610 idx_18488 idx_65760\\t16696', 'long-term-care-insurance\\tidx_3815 idx_604 idx_605 idx_891 idx_136 idx_5293 idx_65761\\t10277']\n"
     ]
    }
   ],
   "source": [
    "print(\" -- Questions -- \")\n",
    "print(question_lines[:2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " -- Answers -- \n",
      "['1\\tidx_1 idx_2 idx_3 idx_4 idx_5 idx_6 idx_7 idx_8 idx_9 idx_10 idx_11 idx_12 idx_13 idx_14 idx_3 idx_12 idx_15 idx_16 idx_17 idx_8 idx_18 idx_19 idx_20 idx_21 idx_3 idx_12 idx_14 idx_22 idx_20 idx_23 idx_24 idx_25 idx_26 idx_27 idx_28 idx_29 idx_8 idx_30 idx_19 idx_11 idx_4 idx_31 idx_32 idx_22 idx_33 idx_34 idx_35 idx_36 idx_37 idx_30 idx_38 idx_39 idx_11 idx_40 idx_41 idx_42 idx_43 idx_44 idx_22 idx_45 idx_46 idx_11 idx_47 idx_48 idx_49 idx_18 idx_50 idx_20 idx_44 idx_22 idx_51 idx_14 idx_52 idx_53 idx_22 idx_40 idx_21 idx_3 idx_54 idx_46 idx_11 idx_55 idx_56 idx_57 idx_58 idx_59 idx_60 idx_61 idx_62 idx_8 idx_50 idx_11 idx_45 idx_63 idx_64 idx_3 idx_65 idx_66 idx_3 idx_67 idx_68 idx_69 idx_70 idx_14 idx_3 idx_71 idx_72 idx_73 idx_21 idx_74 idx_5 idx_75 idx_76 idx_12 idx_8 idx_77 idx_78 idx_15 idx_79 idx_49 idx_18 idx_19 idx_11 idx_54 idx_44 idx_22 idx_80 idx_53 idx_3 idx_12 idx_21 idx_81 idx_54 idx_14 idx_82 idx_83 idx_41 idx_3 idx_84 idx_22 idx_54 idx_23 idx_85 idx_11 idx_86 idx_87 idx_88 idx_89 idx_3 idx_90 idx_41 idx_3 idx_47 idx_12 idx_20 idx_77 idx_91 idx_22 idx_92 idx_93 idx_94 idx_82 idx_14 idx_95 idx_96 idx_97 idx_3 idx_98 idx_99 idx_100 idx_101 idx_3 idx_102 idx_103 idx_104 idx_105 idx_106 idx_107 idx_108 idx_14 idx_109 idx_110 idx_30 idx_111 idx_3 idx_92 idx_112 idx_3 idx_68 idx_113 idx_114 idx_115 idx_52 idx_116 idx_11 idx_10 idx_113 idx_117 idx_118 idx_119 idx_77 idx_120 idx_121 idx_122 idx_41 idx_11 idx_123', '2\\tidx_124 idx_107 idx_11 idx_125 idx_126 idx_127 idx_128 idx_129 idx_81 idx_8 idx_130 idx_30 idx_131 idx_129 idx_97 idx_22 idx_132 idx_133 idx_134 idx_135 idx_41 idx_136 idx_23 idx_130 idx_82 idx_137 idx_138 idx_3 idx_139 idx_140 idx_90 idx_97 idx_141 idx_142 idx_143 idx_3 idx_144 idx_14 idx_97 idx_145 idx_3 idx_146 idx_49 idx_147 idx_148 idx_30 idx_149 idx_119 idx_97 idx_22 idx_150 idx_76 idx_151 idx_81 idx_22 idx_152 idx_153 idx_154 idx_39 idx_155 idx_156 idx_157 idx_3 idx_54 idx_107 idx_61 idx_11 idx_158 idx_159 idx_97 idx_3 idx_160 idx_161 idx_162 idx_163 idx_107 idx_81 idx_119 idx_164 idx_11 idx_165 idx_166 idx_167 idx_81 idx_168 idx_30 idx_3 idx_157 idx_41 idx_3 idx_169 idx_14 idx_170 idx_171 idx_172 idx_82 idx_3 idx_173 idx_174 idx_124 idx_107 idx_175 idx_176 idx_61 idx_3 idx_59 idx_177 idx_81 idx_107 idx_178 idx_179 idx_180 idx_74 idx_181 idx_23 idx_182 idx_97 idx_183 idx_184 idx_185 idx_87 idx_186 idx_187 idx_188 idx_187 idx_189 idx_190 idx_179 idx_191 idx_30 idx_192 idx_193 idx_3 idx_194 idx_164 idx_195 idx_87 idx_196 idx_89 idx_197 idx_198 idx_199 idx_41 idx_3 idx_200 idx_3 idx_201 idx_202 idx_203 idx_204 idx_205 idx_206 idx_3 idx_201 idx_30 idx_207 idx_66 idx_208 idx_209 idx_14 idx_210 idx_211 idx_212 idx_213 idx_212 idx_214 idx_215 idx_216 idx_178 idx_217 idx_218 idx_107 idx_219 idx_57 idx_220 idx_59 idx_8 idx_23 idx_221 idx_11 idx_222 idx_97 idx_171 idx_223 idx_224 idx_23 idx_225 idx_24 idx_97 idx_226 idx_70 idx_41 idx_3 idx_169 idx_193 idx_179 idx_227 idx_82 idx_22 idx_228 idx_87 idx_97 idx_229 idx_230 idx_231 idx_232 idx_82 idx_233 idx_234 idx_3 idx_235 idx_41 idx_140 idx_3 idx_54 idx_230 idx_231 idx_236 idx_14 idx_171 idx_71 idx_41 idx_237 idx_193 idx_179 idx_238 idx_239 idx_3 idx_240 idx_24 idx_3 idx_59 idx_8 idx_241 idx_130 idx_3 idx_242 idx_30 idx_243 idx_177 idx_244 idx_191 idx_11 idx_36 idx_245 idx_14 idx_171 idx_246 idx_30 idx_247 idx_3 idx_248 idx_245 idx_14 idx_249 idx_3 idx_250 idx_7 idx_8 idx_251 idx_81 idx_252 idx_231 idx_253 idx_254 idx_255 idx_8 idx_130 idx_3 idx_242 idx_256 idx_30 idx_257 idx_11 idx_258 idx_116 idx_3 idx_259 idx_7 idx_8 idx_241 idx_260 idx_24 idx_261 idx_30 idx_22 idx_262 idx_104 idx_263 idx_179 idx_264 idx_8 idx_30 idx_265 idx_3 idx_266 idx_87 idx_130 idx_8 idx_267 idx_268 idx_269 idx_81 idx_270 idx_271 idx_99 idx_18 idx_272 idx_8 idx_179 idx_230 idx_16 idx_273 idx_82 idx_3 idx_274 idx_81 idx_164 idx_275 idx_254 idx_3 idx_276 idx_57 idx_277 idx_179 idx_23 idx_130 idx_231 idx_278 idx_97 idx_11 idx_279 idx_280 idx_14 idx_93 idx_130 idx_230 idx_30 idx_281 idx_161 idx_282 idx_41 idx_22 idx_152 idx_283 idx_284 idx_128 idx_129 idx_97 idx_22 idx_285 idx_14 idx_99 idx_286 idx_218 idx_287 idx_288 idx_97 idx_289 idx_290 idx_8 idx_97 idx_291']\n"
     ]
    }
   ],
   "source": [
    "print(\" -- Answers -- \")\n",
    "print(answer_lines[:2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "id2line = {}\n",
    "for line in vocab_lines:\n",
    "    _line = line.split('\\t')\n",
    "    if len(_line) == 2:\n",
    "        id2line[_line[0]] = _line[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating the word tokens for both questions and answers, along with the mapping of the answers enlisted for questions\n",
    "convs, ansid = [] , []\n",
    "for line in question_lines[:-1]:\n",
    "    _line = line.split('\\t')\n",
    "    ansid.append(_line[2].split(' '))\n",
    "    convs.append(_line[1])\n",
    "    \n",
    "convs1 = [ ]\n",
    "for line in answer_lines[:-1]:\n",
    "    _line = line.split('\\t')\n",
    "    convs1.append(_line[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['idx_1285 idx_1010 idx_467 idx_47610 idx_18488 idx_65760', 'idx_3815 idx_604 idx_605 idx_891 idx_136 idx_5293 idx_65761']\n"
     ]
    }
   ],
   "source": [
    "print(convs[:2])  # word tokens present in the question"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['16696'], ['10277']]\n"
     ]
    }
   ],
   "source": [
    "print(ansid[:2])  # answers IDs mapped to the questions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['idx_1 idx_2 idx_3 idx_4 idx_5 idx_6 idx_7 idx_8 idx_9 idx_10 idx_11 idx_12 idx_13 idx_14 idx_3 idx_12 idx_15 idx_16 idx_17 idx_8 idx_18 idx_19 idx_20 idx_21 idx_3 idx_12 idx_14 idx_22 idx_20 idx_23 idx_24 idx_25 idx_26 idx_27 idx_28 idx_29 idx_8 idx_30 idx_19 idx_11 idx_4 idx_31 idx_32 idx_22 idx_33 idx_34 idx_35 idx_36 idx_37 idx_30 idx_38 idx_39 idx_11 idx_40 idx_41 idx_42 idx_43 idx_44 idx_22 idx_45 idx_46 idx_11 idx_47 idx_48 idx_49 idx_18 idx_50 idx_20 idx_44 idx_22 idx_51 idx_14 idx_52 idx_53 idx_22 idx_40 idx_21 idx_3 idx_54 idx_46 idx_11 idx_55 idx_56 idx_57 idx_58 idx_59 idx_60 idx_61 idx_62 idx_8 idx_50 idx_11 idx_45 idx_63 idx_64 idx_3 idx_65 idx_66 idx_3 idx_67 idx_68 idx_69 idx_70 idx_14 idx_3 idx_71 idx_72 idx_73 idx_21 idx_74 idx_5 idx_75 idx_76 idx_12 idx_8 idx_77 idx_78 idx_15 idx_79 idx_49 idx_18 idx_19 idx_11 idx_54 idx_44 idx_22 idx_80 idx_53 idx_3 idx_12 idx_21 idx_81 idx_54 idx_14 idx_82 idx_83 idx_41 idx_3 idx_84 idx_22 idx_54 idx_23 idx_85 idx_11 idx_86 idx_87 idx_88 idx_89 idx_3 idx_90 idx_41 idx_3 idx_47 idx_12 idx_20 idx_77 idx_91 idx_22 idx_92 idx_93 idx_94 idx_82 idx_14 idx_95 idx_96 idx_97 idx_3 idx_98 idx_99 idx_100 idx_101 idx_3 idx_102 idx_103 idx_104 idx_105 idx_106 idx_107 idx_108 idx_14 idx_109 idx_110 idx_30 idx_111 idx_3 idx_92 idx_112 idx_3 idx_68 idx_113 idx_114 idx_115 idx_52 idx_116 idx_11 idx_10 idx_113 idx_117 idx_118 idx_119 idx_77 idx_120 idx_121 idx_122 idx_41 idx_11 idx_123', 'idx_124 idx_107 idx_11 idx_125 idx_126 idx_127 idx_128 idx_129 idx_81 idx_8 idx_130 idx_30 idx_131 idx_129 idx_97 idx_22 idx_132 idx_133 idx_134 idx_135 idx_41 idx_136 idx_23 idx_130 idx_82 idx_137 idx_138 idx_3 idx_139 idx_140 idx_90 idx_97 idx_141 idx_142 idx_143 idx_3 idx_144 idx_14 idx_97 idx_145 idx_3 idx_146 idx_49 idx_147 idx_148 idx_30 idx_149 idx_119 idx_97 idx_22 idx_150 idx_76 idx_151 idx_81 idx_22 idx_152 idx_153 idx_154 idx_39 idx_155 idx_156 idx_157 idx_3 idx_54 idx_107 idx_61 idx_11 idx_158 idx_159 idx_97 idx_3 idx_160 idx_161 idx_162 idx_163 idx_107 idx_81 idx_119 idx_164 idx_11 idx_165 idx_166 idx_167 idx_81 idx_168 idx_30 idx_3 idx_157 idx_41 idx_3 idx_169 idx_14 idx_170 idx_171 idx_172 idx_82 idx_3 idx_173 idx_174 idx_124 idx_107 idx_175 idx_176 idx_61 idx_3 idx_59 idx_177 idx_81 idx_107 idx_178 idx_179 idx_180 idx_74 idx_181 idx_23 idx_182 idx_97 idx_183 idx_184 idx_185 idx_87 idx_186 idx_187 idx_188 idx_187 idx_189 idx_190 idx_179 idx_191 idx_30 idx_192 idx_193 idx_3 idx_194 idx_164 idx_195 idx_87 idx_196 idx_89 idx_197 idx_198 idx_199 idx_41 idx_3 idx_200 idx_3 idx_201 idx_202 idx_203 idx_204 idx_205 idx_206 idx_3 idx_201 idx_30 idx_207 idx_66 idx_208 idx_209 idx_14 idx_210 idx_211 idx_212 idx_213 idx_212 idx_214 idx_215 idx_216 idx_178 idx_217 idx_218 idx_107 idx_219 idx_57 idx_220 idx_59 idx_8 idx_23 idx_221 idx_11 idx_222 idx_97 idx_171 idx_223 idx_224 idx_23 idx_225 idx_24 idx_97 idx_226 idx_70 idx_41 idx_3 idx_169 idx_193 idx_179 idx_227 idx_82 idx_22 idx_228 idx_87 idx_97 idx_229 idx_230 idx_231 idx_232 idx_82 idx_233 idx_234 idx_3 idx_235 idx_41 idx_140 idx_3 idx_54 idx_230 idx_231 idx_236 idx_14 idx_171 idx_71 idx_41 idx_237 idx_193 idx_179 idx_238 idx_239 idx_3 idx_240 idx_24 idx_3 idx_59 idx_8 idx_241 idx_130 idx_3 idx_242 idx_30 idx_243 idx_177 idx_244 idx_191 idx_11 idx_36 idx_245 idx_14 idx_171 idx_246 idx_30 idx_247 idx_3 idx_248 idx_245 idx_14 idx_249 idx_3 idx_250 idx_7 idx_8 idx_251 idx_81 idx_252 idx_231 idx_253 idx_254 idx_255 idx_8 idx_130 idx_3 idx_242 idx_256 idx_30 idx_257 idx_11 idx_258 idx_116 idx_3 idx_259 idx_7 idx_8 idx_241 idx_260 idx_24 idx_261 idx_30 idx_22 idx_262 idx_104 idx_263 idx_179 idx_264 idx_8 idx_30 idx_265 idx_3 idx_266 idx_87 idx_130 idx_8 idx_267 idx_268 idx_269 idx_81 idx_270 idx_271 idx_99 idx_18 idx_272 idx_8 idx_179 idx_230 idx_16 idx_273 idx_82 idx_3 idx_274 idx_81 idx_164 idx_275 idx_254 idx_3 idx_276 idx_57 idx_277 idx_179 idx_23 idx_130 idx_231 idx_278 idx_97 idx_11 idx_279 idx_280 idx_14 idx_93 idx_130 idx_230 idx_30 idx_281 idx_161 idx_282 idx_41 idx_22 idx_152 idx_283 idx_284 idx_128 idx_129 idx_97 idx_22 idx_285 idx_14 idx_99 idx_286 idx_218 idx_287 idx_288 idx_97 idx_289 idx_290 idx_8 idx_97 idx_291']\n"
     ]
    }
   ],
   "source": [
    "print(convs1[:2])  # word tokens present in the answer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Creating matching pair between questions and answers on the basis of the ID allocated to each.\n",
    "\n",
    "questions, answers = [], []\n",
    "for a in range(len(ansid)):\n",
    "      for b in range(len(ansid[a])):\n",
    "            questions.append(convs[a])\n",
    "\n",
    "for a in range(len(ansid)):\n",
    "      for b in range(len(ansid[a])):\n",
    "            answers.append(convs1[int(ansid[a][b])-1])\n",
    "\n",
    "ques, ans =[], []\n",
    "\n",
    "m=0\n",
    "while m<len(questions):\n",
    "       i=0\n",
    "       a=[]\n",
    "       while i < (len(questions[m].split(' '))):\n",
    "            a.append(id2line[questions[m].split(' ')[i]])\n",
    "            i=i+1\n",
    "       ques.append(' '.join(a))\n",
    "       m=m+1\n",
    "\n",
    "n=0\n",
    "while n<len(answers):  \n",
    "        j=0\n",
    "        b=[]\n",
    "        while j < (len(answers[n].split(' '))):\n",
    "            b.append(id2line[answers[n].split(' ')[j]])\n",
    "            j=j+1\n",
    "        ans.append(' '.join(b))\n",
    "        n=n+1     "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "What Does Medicare IME Stand For?\n",
      "According to the Centers for Medicare and Medicaid Services website, cms.gov, IME stands for Indirect Medical Education and is in regards to payment calculation adjustments for a Medicare discharge of higher cost patients receiving care from teaching hospitals relative to non-teaching hospitals. I would recommend contacting CMS to get more information about IME\n",
      "---\n",
      "Is Long Term Care Insurance Tax Free?\n",
      "As a rule, if you buy a tax qualified long term care insurance policy (as nearly all are, these days), and if you are paying the premium yourself, there are tax advantages you will receive. If you are self employed, the entire premium is tax deductible. If working somewhere but paying your own premium for an individual or group policy, you can deduct the premium as a medical expense under the same IRS rules as apply to all medical expenses. In both situations, you also receive the benefits from the policy tax free, if they are ever needed.\n",
      "---\n",
      "Can Husband Drop Wife From Health Insurance?\n",
      "Can a spouse drop another spouse from health insurance? Usually not without the spouse's who is being dropped consent in writting. Most employers who have a quality HR department will require a paper trial for any changes in an employee's benefit plan. When changes are attempted that could come back to haunt the employer, steps are usually taken to comfirm something like this.\n",
      "---\n",
      "Is Medicare Run By The Government?\n",
      "Medicare Part A and Part B is provided by the Federal government for Americans who are 65 and older who have worked and paid Social Security taxes into the system. Medicare is also available to people under the age of 65 that have certain disabilities and people with End-Stage Renal Disease (ESRD).\n",
      "---\n",
      "Is Medicare Run By The Government?\n",
      "Definitely. It is ran by the Center for Medicare and Medicaid Services, a Government Agency given the responsibility of overseeing and administering Medicare and Medicaid. Even Medicare Advantage Plans, which are administered by private insurance companies are strongly regulated by CMMS. They work along with Social Security and Jobs and Family Services to insure that your benefits are available and properly administered.\n",
      "---\n"
     ]
    }
   ],
   "source": [
    "# Printing top 5 questions along with their answers\n",
    "limit = 0\n",
    "for i in range(limit, limit+5):\n",
    "    print(ques[i])\n",
    "    print(ans[i])\n",
    "    print(\"---\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "27987\n",
      "27987\n"
     ]
    }
   ],
   "source": [
    "# Checking the count of the total number of questions and answers\n",
    "print(len(questions))\n",
    "print(len(answers))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Processing Text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "def clean_text(text):\n",
    "\t\"\"\"Cleaning the text by replacing the abbreviated words with their proper full replacement\"\"\"\n",
    "    \n",
    "\ttext = text.lower()\n",
    "\n",
    "\ttext = re.sub(r\"i'm\", \"i am\", text)\n",
    "\ttext = re.sub(r\"he's\", \"he is\", text)\n",
    "\ttext = re.sub(r\"she's\", \"she is\", text)\n",
    "\ttext = re.sub(r\"it's\", \"it is\", text)\n",
    "\ttext = re.sub(r\"that's\", \"that is\", text)\n",
    "\ttext = re.sub(r\"what's\", \"that is\", text)\n",
    "\ttext = re.sub(r\"where's\", \"where is\", text)\n",
    "\ttext = re.sub(r\"how's\", \"how is\", text)\n",
    "\ttext = re.sub(r\"\\'ll\", \" will\", text)\n",
    "\ttext = re.sub(r\"\\'ve\", \" have\", text)\n",
    "\ttext = re.sub(r\"\\'re\", \" are\", text)\n",
    "\ttext = re.sub(r\"\\'d\", \" would\", text)\n",
    "\ttext = re.sub(r\"\\'re\", \" are\", text)\n",
    "\ttext = re.sub(r\"won't\", \"will not\", text)\n",
    "\ttext = re.sub(r\"can't\", \"cannot\", text)\n",
    "\ttext = re.sub(r\"n't\", \" not\", text)\n",
    "\ttext = re.sub(r\"n'\", \"ng\", text)\n",
    "\ttext = re.sub(r\"'bout\", \"about\", text)\n",
    "\ttext = re.sub(r\"'til\", \"until\", text)\n",
    "\ttext = re.sub(r\"[-()\\\"#/@;:<>{}`+=~|.!?,']\", \"\", text)\n",
    "\n",
    "\treturn text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Applying the 'clean_text()' function on the set of Questions and Answers\n",
    "clean_questions = []\n",
    "for question in ques:\n",
    "    clean_questions.append(clean_text(question))\n",
    "    \n",
    "clean_answers = []    \n",
    "for answer in ans:\n",
    "    clean_answers.append(clean_text(answer))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "what does medicare ime stand for\n",
      "according to the centers for medicare and medicaid services website cmsgov ime stands for indirect medical education and is in regards to payment calculation adjustments for a medicare discharge of higher cost patients receiving care from teaching hospitals relative to nonteaching hospitals i would recommend contacting cms to get more information about ime\n",
      "----\n",
      "is long term care insurance tax free\n",
      "as a rule if you buy a tax qualified long term care insurance policy as nearly all are these days and if you are paying the premium yourself there are tax advantages you will receive if you are self employed the entire premium is tax deductible if working somewhere but paying your own premium for an individual or group policy you can deduct the premium as a medical expense under the same irs rules as apply to all medical expenses in both situations you also receive the benefits from the policy tax free if they are ever needed\n",
      "----\n",
      "can husband drop wife from health insurance\n",
      "can a spouse drop another spouse from health insurance usually not without the spouses who is being dropped consent in writting most employers who have a quality hr department will require a paper trial for any changes in an employees benefit plan when changes are attempted that could come back to haunt the employer steps are usually taken to comfirm something like this\n",
      "----\n",
      "is medicare run by the government\n",
      "medicare part a and part b is provided by the federal government for americans who are 65 and older who have worked and paid social security taxes into the system medicare is also available to people under the age of 65 that have certain disabilities and people with endstage renal disease esrd\n",
      "----\n",
      "is medicare run by the government\n",
      "definitely it is ran by the center for medicare and medicaid services a government agency given the responsibility of overseeing and administering medicare and medicaid even medicare advantage plans which are administered by private insurance companies are strongly regulated by cmms they work along with social security and jobs and family services to insure that your benefits are available and properly administered\n",
      "----\n"
     ]
    }
   ],
   "source": [
    "limit = 0\n",
    "for i in range(limit, limit+5):\n",
    "    print(clean_questions[i])\n",
    "    print(clean_answers[i])\n",
    "    print('----')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "lengths = []\n",
    "for question in clean_questions:\n",
    "    lengths.append(len(question.split()))\n",
    "for answer in clean_answers:\n",
    "    lengths.append(len(answer.split()))\n",
    "\n",
    "# Create a dataframe so that the values can be inspected\n",
    "lengths = pd.DataFrame(lengths, columns=['counts'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>counts</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>55974.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>54.176725</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>67.638972</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>2.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0%</th>\n",
       "      <td>2.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>7.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>30.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>78.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85%</th>\n",
       "      <td>103.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90%</th>\n",
       "      <td>126.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95%</th>\n",
       "      <td>173.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99%</th>\n",
       "      <td>314.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1176.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             counts\n",
       "count  55974.000000\n",
       "mean      54.176725\n",
       "std       67.638972\n",
       "min        2.000000\n",
       "0%         2.000000\n",
       "25%        7.000000\n",
       "50%       30.000000\n",
       "75%       78.000000\n",
       "85%      103.000000\n",
       "90%      126.000000\n",
       "95%      173.000000\n",
       "99%      314.000000\n",
       "max     1176.000000"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lengths.describe(percentiles=[0,0.25,0.5,0.75,0.85,0.9,0.95,0.99])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Remove questions and answers that are shorter than 1 words and longer than 100 words.\n",
    "min_line_length, max_line_length = 2, 100\n",
    "\n",
    "# Filter out the questions that are too short/long\n",
    "short_questions_temp, short_answers_temp = [], []\n",
    "\n",
    "i = 0\n",
    "for question in clean_questions:\n",
    "    if len(question.split()) >= min_line_length and len(question.split()) <= max_line_length:\n",
    "        short_questions_temp.append(question)\n",
    "        short_answers_temp.append(clean_answers[i])\n",
    "    i += 1\n",
    "\n",
    "# Filter out the answers that are too short/long\n",
    "short_questions, short_answers = [], []\n",
    "\n",
    "i = 0\n",
    "for answer in short_answers_temp:\n",
    "    if len(answer.split()) >= min_line_length and len(answer.split()) <= max_line_length:\n",
    "        short_answers.append(answer)\n",
    "        short_questions.append(short_questions_temp[i])\n",
    "    i += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# of questions: 19108\n",
      "# of answers: 19108\n",
      "% of data used: 68.27%\n"
     ]
    }
   ],
   "source": [
    "print(\"# of questions:\", len(short_questions))\n",
    "print(\"# of answers:\", len(short_answers))\n",
    "print(\"% of data used: {}%\".format(round(len(short_questions)/len(questions),4)*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def pad_sentence_batch(sentence_batch, vocab_to_int):\n",
    "\"\"\"Including <PAD> token in sentence to make all batches of same length\"\"\"\n",
    "    max_sentence = max([len(sentence) for sentence in sentence_batch])\n",
    "    return [sentence + [vocab_to_int['<PAD>']] * (max_sentence - len(sentence)) for sentence in sentence_batch]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Create a dictionary for the frequency of the vocabulary\n",
    "vocab = {}\n",
    "for question in short_questions:\n",
    "    for word in question.split():\n",
    "        if word not in vocab:\n",
    "            vocab[word] = 1\n",
    "        else:\n",
    "            vocab[word] += 1\n",
    "            \n",
    "for answer in short_answers:\n",
    "    for word in answer.split():\n",
    "        if word not in vocab:\n",
    "            vocab[word] = 1\n",
    "        else:\n",
    "            vocab[word] += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Remove rare words from the vocabulary.\n",
    "threshold = 1\n",
    "count = 0\n",
    "for k,v in vocab.items():\n",
    "    if v >= threshold:\n",
    "        count += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Size of total vocab: 18983\n",
      "Size of vocab we will use: 18983\n"
     ]
    }
   ],
   "source": [
    "print(\"Size of total vocab:\", len(vocab))\n",
    "print(\"Size of vocab we will use:\", count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Create dictionaries to provide a unique integer for each word.\n",
    "questions_vocab_to_int = {}\n",
    "\n",
    "word_num = 0\n",
    "for word, count in vocab.items():\n",
    "    if count >= threshold:\n",
    "        questions_vocab_to_int[word] = word_num\n",
    "        word_num += 1\n",
    "        \n",
    "answers_vocab_to_int = {}\n",
    "\n",
    "word_num = 0\n",
    "for word, count in vocab.items():\n",
    "    if count >= threshold:\n",
    "        answers_vocab_to_int[word] = word_num\n",
    "        word_num += 1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Adding unique tokens to the present vocabulary\n",
    "codes = ['<PAD>','<EOS>','<UNK>','<GO>']\n",
    "\n",
    "for code in codes:\n",
    "    questions_vocab_to_int[code] = len(questions_vocab_to_int)+1\n",
    "    \n",
    "for code in codes:\n",
    "    answers_vocab_to_int[code] = len(answers_vocab_to_int)+1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Creating dictionary so as to map the integers to their respective words, inverse of vocab_to_int\n",
    "questions_int_to_vocab = {v_i: v for v, v_i in questions_vocab_to_int.items()}\n",
    "answers_int_to_vocab = {v_i: v for v, v_i in answers_vocab_to_int.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "18987\n",
      "18987\n",
      "18987\n",
      "18987\n"
     ]
    }
   ],
   "source": [
    "print(len(questions_vocab_to_int))\n",
    "print(len(questions_int_to_vocab))\n",
    "print(len(answers_vocab_to_int))\n",
    "print(len(answers_int_to_vocab))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Convert the text to integers, and replacing any of the words not present in the respective vocabulary with <UNK> token \n",
    "questions_int = []\n",
    "for question in short_questions:\n",
    "    ints = []\n",
    "    for word in question.split():\n",
    "        if word not in questions_vocab_to_int:\n",
    "            ints.append(questions_vocab_to_int['<UNK>'])\n",
    "        else:\n",
    "            ints.append(questions_vocab_to_int[word])\n",
    "    questions_int.append(ints)\n",
    "    \n",
    "answers_int = []\n",
    "for answer in short_answers:\n",
    "    ints = []\n",
    "    for word in answer.split():\n",
    "        if word not in answers_vocab_to_int:\n",
    "            ints.append(answers_vocab_to_int['<UNK>'])\n",
    "        else:\n",
    "            ints.append(answers_vocab_to_int[word])\n",
    "    answers_int.append(ints)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of words: 1450824\n",
      "Number of times <UNK> is used: 0\n",
      "Percent of words that are <UNK>: 0.0%\n"
     ]
    }
   ],
   "source": [
    "# Calculate what percentage of all words have been replaced with <UNK>\n",
    "word_count = 0\n",
    "unk_count = 0\n",
    "\n",
    "for question in questions_int:\n",
    "    for word in question:\n",
    "        if word == questions_vocab_to_int[\"<UNK>\"]:\n",
    "            unk_count += 1\n",
    "        word_count += 1\n",
    "    \n",
    "for answer in answers_int:\n",
    "    for word in answer:\n",
    "        if word == answers_vocab_to_int[\"<UNK>\"]:\n",
    "            unk_count += 1\n",
    "        word_count += 1\n",
    "    \n",
    "unk_ratio = round(unk_count/word_count,4)*100\n",
    "    \n",
    "print(\"Total number of words:\", word_count)\n",
    "print(\"Number of times <UNK> is used:\", unk_count)\n",
    "print(\"Percent of words that are <UNK>: {}%\".format(round(unk_ratio,3)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "19108\n",
      "19108\n",
      "19108\n",
      "19108\n",
      "\n",
      "[219, 13]\n",
      "[219, 13, 58, 2310, 3636, 1384, 3365, 220, 2332, 219, 13, 243, 55, 394, 166, 219, 13, 243, 55, 72, 76, 135, 1140, 18574, 5, 4624, 26, 1651, 573, 175, 21, 2363, 193, 76, 4126, 312, 811, 105, 18575, 7875, 175, 1514, 193, 18576, 135, 21, 2363, 26, 21, 3636, 219, 13, 21, 2363, 219, 171]\n",
      "why can\n",
      "why can a simple question but yet so complex why can someone do this or why can someone do that i have often pondered for hours to come up with the answer and i believe after years of thoughtprovoking consultation with friends and relativesi have the answer to the question why can the answer why not\n",
      "\n",
      "[133, 479, 56]\n",
      "[242, 4123, 3646, 282, 306, 56, 2627, 3924, 201, 918, 5047, 4443, 5, 176, 56, 5048, 193, 21, 2346, 105, 10, 176, 763, 1166, 176, 56, 201, 1916, 763, 193, 1651, 299, 21, 4273, 105, 5049, 5050, 1384, 3685, 84, 6, 29, 202, 10, 3602, 237, 2458, 84, 2379, 299, 21, 2346, 105, 10, 288, 5048, 56, 201, 1168, 20, 21, 2346, 105, 10, 32, 669, 153]\n",
      "who governs annuities\n",
      "if youre asking about all annuities then here are two governing bodies for variable annuities finra and the department of insurance variable products like variable annuities are registered products and come under the oversight of finras jurisdiction but because it is an annuity insurance product as well it falls under the department of insurance non finra annuities are governed by the department of insurance in each state\n",
      "\n",
      "[0, 201, 56]\n",
      "[29, 202, 6, 29, 10, 3602, 58, 36, 10, 179, 4187, 93, 17, 1452, 1989, 409, 29, 202, 4187, 93, 17, 951, 1989, 7, 56, 201, 2332, 4642, 32, 1668, 5, 58, 218, 105, 43, 3707, 395, 166, 32, 5087, 21, 181, 81, 122, 21, 4062, 58, 3272, 320, 1621, 881, 5, 21, 36, 105, 21, 4062, 388, 201, 386, 949, 105, 394, 101, 787, 56, 201, 170, 5088, 17, 643, 771]\n",
      "what are annuities\n",
      "an annuity is an insurance product a life insurance policy protects you from dying too soon an annuity protects you from living too long annuities are complex basically in exchange for a sum of money either immediate or in installments the company will pay the annuitant a specific amount normally monthly for the life of the annuitant there are many modifications of this basic form annuities are taxed differently from other programs\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Next, sorting the questions and answers on basis of the length of the questions. \n",
    "# This exercise will reduce the amount of padding being done during the training process.\n",
    "# This will speed up the training process and reduce the training loss.\n",
    "\n",
    "sorted_questions = []\n",
    "short_questions1 = []\n",
    "sorted_answers = []\n",
    "short_answers1= []\n",
    "\n",
    "\n",
    "for length in range(1, max_line_length+1):\n",
    "    for i in enumerate(questions_int):\n",
    "        if len(i[1]) == length:\n",
    "            sorted_questions.append(questions_int[i[0]])\n",
    "            short_questions1.append(short_questions[i[0]])\n",
    "            sorted_answers.append(answers_int[i[0]])\n",
    "            short_answers1.append(short_answers[i[0]])\n",
    "            \n",
    "\n",
    "print(len(sorted_questions))\n",
    "print(len(sorted_answers))\n",
    "print(len(short_questions1))\n",
    "print(len(short_answers1))\n",
    "print()\n",
    "for i in range(3):\n",
    "    print(sorted_questions[i])\n",
    "    print(sorted_answers[i])\n",
    "    print(short_questions1[i])\n",
    "    print(short_answers1[i])\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[37, 6, 36, 10, 466]\n",
      "how is life insurance used\n",
      "[8, 36, 10, 6, 466, 26, 626, 58, 199, 200, 1130, 58, 3512, 31, 105, 208, 601, 10, 6, 466, 26, 626, 58, 199, 200, 47, 402, 208, 21, 179, 6, 32, 1259, 32, 1429, 26, 4210, 394, 193, 135, 249, 132, 137, 3608, 1135, 1044, 3472, 1044, 201, 3002, 26, 2133, 21, 179, 26, 1986, 1914, 10565, 2504, 3548, 3472, 1044, 3608, 11, 12, 193, 1184, 21, 179, 21, 4193, 105, 13628, 386, 622, 17, 3057, 3007, 26, 3615, 7, 8, 9]\n",
      "term life insurance is used to provide a death benefit during a specified period of time permanent insurance is used to provide a death benefit at any time the policy is in force in order to accomplish this and have level premiums policies accumulate extra funds these funds are designed to allow the policy to meet its lifelong obligations however these funds accumulate tax free and give the policy the potential of solving many problems from funding education to providing long term care\n"
     ]
    }
   ],
   "source": [
    "print(sorted_questions[1547])\n",
    "print(short_questions1[1547])\n",
    "print(sorted_answers[1547])\n",
    "print(short_answers1[1547])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Seq2Seq helper functions for Encoder and Decoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def model_inputs():\n",
    "    input_data = tf.placeholder(tf.int32, [None, None], name='input')\n",
    "    targets = tf.placeholder(tf.int32, [None, None], name='targets')\n",
    "    lr = tf.placeholder(tf.float32, name='learning_rate')\n",
    "    keep_prob = tf.placeholder(tf.float32, name='keep_prob')\n",
    "\n",
    "    return input_data, targets, lr, keep_prob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def process_encoding_input(target_data, vocab_to_int, batch_size):\n",
    "    ending = tf.strided_slice(target_data, [0, 0], [batch_size, -1], [1, 1])\n",
    "    dec_input = tf.concat([tf.fill([batch_size, 1], vocab_to_int['<GO>']), ending], 1)\n",
    "\n",
    "    return dec_input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def encoding_layer(rnn_inputs, rnn_size, num_layers, keep_prob, sequence_length):\n",
    "    lstm = tf.contrib.rnn.BasicLSTMCell(rnn_size)\n",
    "    drop = tf.contrib.rnn.DropoutWrapper(lstm, input_keep_prob = keep_prob)\n",
    "    enc_cell = tf.contrib.rnn.MultiRNNCell([drop] * num_layers)\n",
    "    _, enc_state = tf.nn.bidirectional_dynamic_rnn(cell_fw = enc_cell, cell_bw = enc_cell,\n",
    "                                                   sequence_length = sequence_length, inputs = rnn_inputs, dtype=tf.float32)\n",
    "    return enc_state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def decoding_layer_train(encoder_state, dec_cell, dec_embed_input, sequence_length, decoding_scope,\n",
    "                         output_fn, keep_prob, batch_size):\n",
    "    \n",
    "    attention_states = tf.zeros([batch_size, 1, dec_cell.output_size])\n",
    "    \n",
    "    att_keys, att_vals, att_score_fn, att_construct_fn = tf.contrib.seq2seq.prepare_attention(attention_states, attention_option=\"bahdanau\", num_units=dec_cell.output_size)\n",
    "    \n",
    "    train_decoder_fn = tf.contrib.seq2seq.attention_decoder_fn_train(encoder_state[0], att_keys, att_vals,  att_score_fn, att_construct_fn,  name = \"attn_dec_train\")\n",
    "    \n",
    "    train_pred, _, _ = tf.contrib.seq2seq.dynamic_rnn_decoder(dec_cell, train_decoder_fn,  dec_embed_input, sequence_length, scope=decoding_scope)\n",
    "    train_pred_drop = tf.nn.dropout(train_pred, keep_prob)\n",
    "    \n",
    "    return output_fn(train_pred_drop)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def decoding_layer_infer(encoder_state, dec_cell, dec_embeddings, start_of_sequence_id, end_of_sequence_id,\n",
    "                         maximum_length, vocab_size, decoding_scope, output_fn, keep_prob, batch_size):\n",
    "    \n",
    "    attention_states = tf.zeros([batch_size, 1, dec_cell.output_size])\n",
    "    \n",
    "    att_keys, att_vals, att_score_fn, att_construct_fn = tf.contrib.seq2seq.prepare_attention(attention_states, attention_option=\"bahdanau\", num_units=dec_cell.output_size)\n",
    "    \n",
    "    infer_decoder_fn = tf.contrib.seq2seq.attention_decoder_fn_inference(output_fn, encoder_state[0],  att_keys, att_vals,  att_score_fn, att_construct_fn, \n",
    "                        dec_embeddings, start_of_sequence_id, end_of_sequence_id, maximum_length, vocab_size, name = \"attn_dec_inf\")\n",
    "    \n",
    "    infer_logits, _, _ = tf.contrib.seq2seq.dynamic_rnn_decoder(dec_cell, infer_decoder_fn, scope=decoding_scope)\n",
    "    \n",
    "    return infer_logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def decoding_layer(dec_embed_input, dec_embeddings, encoder_state, vocab_size, sequence_length, rnn_size,\n",
    "                   num_layers, vocab_to_int, keep_prob, batch_size):\n",
    "    \n",
    "    with tf.variable_scope(\"decoding\") as decoding_scope:\n",
    "        lstm = tf.contrib.rnn.BasicLSTMCell(rnn_size)\n",
    "        drop = tf.contrib.rnn.DropoutWrapper(lstm, input_keep_prob = keep_prob)\n",
    "        dec_cell = tf.contrib.rnn.MultiRNNCell([drop] * num_layers)\n",
    "        \n",
    "        weights = tf.truncated_normal_initializer(stddev=0.1)\n",
    "        biases = tf.zeros_initializer()\n",
    "        output_fn = lambda x: tf.contrib.layers.fully_connected(x, vocab_size, None,  scope=decoding_scope, weights_initializer = weights, biases_initializer = biases)\n",
    "\n",
    "        train_logits = decoding_layer_train(encoder_state, dec_cell,  dec_embed_input, sequence_length,  decoding_scope, output_fn, keep_prob, batch_size)\n",
    "\n",
    "        decoding_scope.reuse_variables()\n",
    "        infer_logits = decoding_layer_infer(encoder_state, dec_cell, dec_embeddings, vocab_to_int['<GO>'], vocab_to_int['<EOS>'], \n",
    "                    sequence_length - 1, vocab_size,  decoding_scope, output_fn, keep_prob, batch_size)\n",
    "\n",
    "    return train_logits, infer_logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def seq2seq_model(input_data, target_data, keep_prob, batch_size, sequence_length, answers_vocab_size, \n",
    "                  questions_vocab_size, enc_embedding_size, dec_embedding_size, rnn_size, num_layers, \n",
    "                  questions_vocab_to_int):\n",
    "    \n",
    "    enc_embed_input = tf.contrib.layers.embed_sequence(input_data, answers_vocab_size+1,  enc_embedding_size, initializer = tf.random_uniform_initializer(0,1))\n",
    "    \n",
    "    enc_state = encoding_layer(enc_embed_input, rnn_size, num_layers, keep_prob, sequence_length)\n",
    "\n",
    "    dec_input = process_encoding_input(target_data, questions_vocab_to_int, batch_size)\n",
    "    dec_embeddings = tf.Variable(tf.random_uniform([questions_vocab_size+1, dec_embedding_size], 0, 1))\n",
    "    dec_embed_input = tf.nn.embedding_lookup(dec_embeddings, dec_input)\n",
    "    \n",
    "    train_logits, infer_logits = decoding_layer(dec_embed_input, dec_embeddings, enc_state, questions_vocab_size, \n",
    "                            sequence_length, rnn_size, num_layers, questions_vocab_to_int,  keep_prob, batch_size)\n",
    "    \n",
    "    return train_logits, infer_logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Setting the model parameters\n",
    "epochs = 50\n",
    "batch_size = 64\n",
    "rnn_size = 512\n",
    "num_layers = 2\n",
    "encoding_embedding_size = 512\n",
    "decoding_embedding_size = 512\n",
    "learning_rate = 0.005\n",
    "learning_rate_decay = 0.9\n",
    "min_learning_rate = 0.0001\n",
    "keep_probability = 0.75"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "# Starting the session\n",
    "sess = tf.InteractiveSession()\n",
    "    \n",
    "# Loading the model inputs    \n",
    "input_data, targets, lr, keep_prob = model_inputs()\n",
    "\n",
    "# Sequence length is max_line_length for each batch\n",
    "sequence_length = tf.placeholder_with_default(max_line_length, None, name='sequence_length')\n",
    "\n",
    "# Finding shape of the input data for sequence_loss\n",
    "input_shape = tf.shape(input_data)\n",
    "\n",
    "# Create the training and inference logits\n",
    "train_logits, inference_logits = seq2seq_model( tf.reverse(input_data, [-1]), targets, keep_prob, batch_size, sequence_length, len(answers_vocab_to_int), \n",
    "    len(questions_vocab_to_int), encoding_embedding_size, decoding_embedding_size, rnn_size, num_layers,  questions_vocab_to_int)\n",
    "\n",
    "# Create inference logits tensor\n",
    "tf.identity(inference_logits, 'logits')\n",
    "\n",
    "with tf.name_scope(\"optimization\"):\n",
    "    # Calculating Loss function\n",
    "    cost = tf.contrib.seq2seq.sequence_loss( train_logits, targets, tf.ones([input_shape[0], sequence_length]))\n",
    "\n",
    "    # Using Adam Optimizer\n",
    "    optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "\n",
    "    # Performing Gradient Clipping to handle the vanishing gradient problem\n",
    "    gradients = optimizer.compute_gradients(cost)\n",
    "    capped_gradients = [(tf.clip_by_value(grad, -5., 5.), var) for grad, var in gradients if grad is not None]\n",
    "    train_op = optimizer.apply_gradients(capped_gradients)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def batch_data(questions, answers, batch_size):\n",
    "\n",
    "    for batch_i in range(0, len(questions)//batch_size):\n",
    "        start_i = batch_i * batch_size\n",
    "        questions_batch = questions[start_i:start_i + batch_size]\n",
    "        answers_batch = answers[start_i:start_i + batch_size]\n",
    "        pad_questions_batch = np.array(pad_sentence_batch(questions_batch, questions_vocab_to_int))\n",
    "        pad_answers_batch = np.array(pad_sentence_batch(answers_batch, answers_vocab_to_int))\n",
    "        yield pad_questions_batch, pad_answers_batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "16242\n",
      "2866\n"
     ]
    }
   ],
   "source": [
    "# Creating train and validation datasets for both questions and answers, with 15% to validation\n",
    "train_valid_split = int(len(sorted_questions)*0.15)\n",
    "\n",
    "train_questions = sorted_questions[train_valid_split:]\n",
    "train_answers = sorted_answers[train_valid_split:]\n",
    "\n",
    "valid_questions = sorted_questions[:train_valid_split]\n",
    "valid_answers = sorted_answers[:train_valid_split]\n",
    "\n",
    "print(len(train_questions))\n",
    "print(len(valid_questions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "display_step = 20        # Check training loss after every 20 batches\n",
    "stop_early = 0 \n",
    "stop = 5                 # If the validation loss decreases after 5 consecutive checks, stop training\n",
    "validation_check = ((len(train_questions))//batch_size//2)-1        # Counter for checking validation loss\n",
    "total_train_loss = 0     # Record the training loss for each display step\n",
    "summary_valid_loss = []     # Record the validation loss for saving improvements in the model\n",
    "\n",
    "checkpoint= \"./best_model.ckpt\"   # creating the checkpoint file in the current directory\n",
    "\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch   1/50 Batch    0/253 - Loss:  0.494, Seconds: 1060.06\n",
      "Epoch   1/50 Batch   20/253 - Loss:  8.450, Seconds: 905.71\n",
      "Epoch   1/50 Batch   40/253 - Loss:  4.540, Seconds: 933.88\n",
      "Epoch   1/50 Batch   60/253 - Loss:  4.401, Seconds: 740.15\n",
      "Epoch   1/50 Batch   80/253 - Loss:  4.453, Seconds: 831.04\n",
      "Epoch   1/50 Batch  100/253 - Loss:  4.338, Seconds: 774.67\n",
      "Epoch   1/50 Batch  120/253 - Loss:  4.295, Seconds: 832.49\n",
      "Valid Loss:  4.091, Seconds: 675.05\n",
      "New Record!\n",
      "Epoch   1/50 Batch  140/253 - Loss:  4.255, Seconds: 822.40\n",
      "Epoch   1/50 Batch  160/253 - Loss:  4.232, Seconds: 888.85\n",
      "Epoch   1/50 Batch  180/253 - Loss:  4.168, Seconds: 858.95\n",
      "Epoch   1/50 Batch  200/253 - Loss:  4.093, Seconds: 849.23\n",
      "Epoch   1/50 Batch  220/253 - Loss:  4.034, Seconds: 846.77\n",
      "Epoch   1/50 Batch  240/253 - Loss:  4.005, Seconds: 809.77\n",
      "Valid Loss:  3.903, Seconds: 509.83\n",
      "New Record!\n",
      "Epoch   2/50 Batch    0/253 - Loss:  2.574, Seconds: 699.78\n",
      "Epoch   2/50 Batch   20/253 - Loss:  3.958, Seconds: 773.99\n",
      "Epoch   2/50 Batch   40/253 - Loss:  3.875, Seconds: 800.03\n",
      "Epoch   2/50 Batch   60/253 - Loss:  3.817, Seconds: 756.63\n",
      "Epoch   2/50 Batch   80/253 - Loss:  3.906, Seconds: 815.33\n",
      "Epoch   2/50 Batch  100/253 - Loss:  3.841, Seconds: 780.41\n",
      "Epoch   2/50 Batch  120/253 - Loss:  3.812, Seconds: 703.94\n",
      "Valid Loss:  3.662, Seconds: 495.80\n",
      "New Record!\n",
      "Epoch   2/50 Batch  140/253 - Loss:  3.785, Seconds: 831.50\n",
      "Epoch   2/50 Batch  160/253 - Loss:  3.768, Seconds: 722.60\n",
      "Epoch   2/50 Batch  180/253 - Loss:  3.723, Seconds: 752.97\n",
      "Epoch   2/50 Batch  200/253 - Loss:  3.679, Seconds: 686.36\n",
      "Epoch   2/50 Batch  220/253 - Loss:  3.656, Seconds: 722.30\n",
      "Epoch   2/50 Batch  240/253 - Loss:  3.647, Seconds: 736.86\n",
      "Valid Loss:  3.632, Seconds: 472.10\n",
      "New Record!\n",
      "Epoch   3/50 Batch    0/253 - Loss:  2.369, Seconds: 676.16\n",
      "Epoch   3/50 Batch   20/253 - Loss:  3.657, Seconds: 769.88\n",
      "Epoch   3/50 Batch   40/253 - Loss:  3.578, Seconds: 692.57\n",
      "Epoch   3/50 Batch   60/253 - Loss:  3.522, Seconds: 699.34\n",
      "Epoch   3/50 Batch   80/253 - Loss:  3.608, Seconds: 717.82\n",
      "Epoch   3/50 Batch  100/253 - Loss:  3.542, Seconds: 682.92\n",
      "Epoch   3/50 Batch  120/253 - Loss:  3.529, Seconds: 659.56\n",
      "Valid Loss:  3.414, Seconds: 468.27\n",
      "New Record!\n",
      "Epoch   3/50 Batch  140/253 - Loss:  3.510, Seconds: 610.44\n",
      "Epoch   3/50 Batch  160/253 - Loss:  3.494, Seconds: 721.53\n",
      "Epoch   3/50 Batch  180/253 - Loss:  3.465, Seconds: 707.51\n",
      "Epoch   3/50 Batch  200/253 - Loss:  3.425, Seconds: 742.86\n",
      "Epoch   3/50 Batch  220/253 - Loss:  3.407, Seconds: 693.83\n",
      "Epoch   3/50 Batch  240/253 - Loss:  3.397, Seconds: 687.35\n",
      "Valid Loss:  3.403, Seconds: 476.70\n",
      "New Record!\n",
      "Epoch   4/50 Batch    0/253 - Loss:  2.218, Seconds: 657.53\n",
      "Epoch   4/50 Batch   20/253 - Loss:  3.404, Seconds: 701.06\n",
      "Epoch   4/50 Batch   40/253 - Loss:  3.335, Seconds: 666.34\n",
      "Epoch   4/50 Batch   60/253 - Loss:  3.287, Seconds: 670.19\n",
      "Epoch   4/50 Batch   80/253 - Loss:  3.380, Seconds: 704.22\n",
      "Epoch   4/50 Batch  100/253 - Loss:  3.323, Seconds: 683.84\n",
      "Epoch   4/50 Batch  120/253 - Loss:  3.310, Seconds: 659.41\n",
      "Valid Loss:  3.235, Seconds: 427.85\n",
      "New Record!\n",
      "Epoch   4/50 Batch  140/253 - Loss:  3.308, Seconds: 737.57\n",
      "Epoch   4/50 Batch  160/253 - Loss:  3.297, Seconds: 681.90\n",
      "Epoch   4/50 Batch  180/253 - Loss:  3.273, Seconds: 726.89\n",
      "Epoch   4/50 Batch  200/253 - Loss:  3.234, Seconds: 695.40\n",
      "Epoch   4/50 Batch  220/253 - Loss:  3.229, Seconds: 708.46\n",
      "Epoch   4/50 Batch  240/253 - Loss:  3.217, Seconds: 695.96\n",
      "Valid Loss:  3.250, Seconds: 467.03\n",
      "No Improvement.\n",
      "Epoch   5/50 Batch    0/253 - Loss:  2.111, Seconds: 681.59\n",
      "Epoch   5/50 Batch   20/253 - Loss:  3.221, Seconds: 714.70\n",
      "Epoch   5/50 Batch   40/253 - Loss:  3.155, Seconds: 632.06\n",
      "Epoch   5/50 Batch   60/253 - Loss:  3.113, Seconds: 661.06\n",
      "Epoch   5/50 Batch   80/253 - Loss:  3.211, Seconds: 702.20\n",
      "Epoch   5/50 Batch  100/253 - Loss:  3.163, Seconds: 662.19\n",
      "Epoch   5/50 Batch  120/253 - Loss:  3.153, Seconds: 653.28\n",
      "Valid Loss:  3.119, Seconds: 436.66\n",
      "New Record!\n",
      "Epoch   5/50 Batch  140/253 - Loss:  3.157, Seconds: 705.01\n",
      "Epoch   5/50 Batch  160/253 - Loss:  3.143, Seconds: 719.78\n",
      "Epoch   5/50 Batch  180/253 - Loss:  3.121, Seconds: 613.95\n",
      "Epoch   5/50 Batch  200/253 - Loss:  3.088, Seconds: 704.71\n",
      "Epoch   5/50 Batch  220/253 - Loss:  3.088, Seconds: 658.94\n",
      "Epoch   5/50 Batch  240/253 - Loss:  3.083, Seconds: 686.02\n",
      "Valid Loss:  3.150, Seconds: 464.55\n",
      "No Improvement.\n",
      "Epoch   6/50 Batch    0/253 - Loss:  2.027, Seconds: 646.65\n",
      "Epoch   6/50 Batch   20/253 - Loss:  3.080, Seconds: 682.20\n",
      "Epoch   6/50 Batch   40/253 - Loss:  3.022, Seconds: 678.49\n",
      "Epoch   6/50 Batch   60/253 - Loss:  2.990, Seconds: 666.95\n",
      "Epoch   6/50 Batch   80/253 - Loss:  3.080, Seconds: 632.96\n",
      "Epoch   6/50 Batch  100/253 - Loss:  3.023, Seconds: 669.13\n",
      "Epoch   6/50 Batch  120/253 - Loss:  3.028, Seconds: 621.96\n",
      "Valid Loss:  3.036, Seconds: 402.64\n",
      "New Record!\n",
      "Epoch   6/50 Batch  140/253 - Loss:  3.035, Seconds: 676.32\n",
      "Epoch   6/50 Batch  160/253 - Loss:  3.023, Seconds: 683.21\n",
      "Epoch   6/50 Batch  180/253 - Loss:  3.006, Seconds: 662.52\n",
      "Epoch   6/50 Batch  200/253 - Loss:  2.980, Seconds: 617.94\n",
      "Epoch   6/50 Batch  220/253 - Loss:  2.982, Seconds: 666.05\n",
      "Epoch   6/50 Batch  240/253 - Loss:  2.976, Seconds: 687.11\n",
      "Valid Loss:  3.073, Seconds: 466.41\n",
      "No Improvement.\n",
      "Epoch   7/50 Batch    0/253 - Loss:  1.962, Seconds: 626.70\n",
      "Epoch   7/50 Batch   20/253 - Loss:  2.973, Seconds: 705.24\n",
      "Epoch   7/50 Batch   40/253 - Loss:  2.924, Seconds: 731.53\n",
      "Epoch   7/50 Batch   60/253 - Loss:  2.885, Seconds: 616.72\n",
      "Epoch   7/50 Batch   80/253 - Loss:  2.975, Seconds: 792.69\n",
      "Epoch   7/50 Batch  100/253 - Loss:  2.922, Seconds: 642.26\n",
      "Epoch   7/50 Batch  120/253 - Loss:  2.930, Seconds: 738.20\n",
      "Valid Loss:  2.983, Seconds: 509.23\n",
      "New Record!\n",
      "Epoch   7/50 Batch  140/253 - Loss:  2.937, Seconds: 721.85\n",
      "Epoch   7/50 Batch  160/253 - Loss:  2.929, Seconds: 806.39\n",
      "Epoch   7/50 Batch  180/253 - Loss:  2.912, Seconds: 651.23\n",
      "Epoch   7/50 Batch  200/253 - Loss:  2.889, Seconds: 819.56\n",
      "Epoch   7/50 Batch  220/253 - Loss:  2.888, Seconds: 744.60\n",
      "Epoch   7/50 Batch  240/253 - Loss:  2.882, Seconds: 882.17\n",
      "Valid Loss:  3.023, Seconds: 532.09\n",
      "No Improvement.\n",
      "Epoch   8/50 Batch    0/253 - Loss:  1.909, Seconds: 683.00\n",
      "Epoch   8/50 Batch   20/253 - Loss:  2.881, Seconds: 770.65\n",
      "Epoch   8/50 Batch   40/253 - Loss:  2.833, Seconds: 793.37\n",
      "Epoch   8/50 Batch   60/253 - Loss:  2.797, Seconds: 746.45\n",
      "Epoch   8/50 Batch   80/253 - Loss:  2.890, Seconds: 871.87\n",
      "Epoch   8/50 Batch  100/253 - Loss:  2.833, Seconds: 859.63\n",
      "Epoch   8/50 Batch  120/253 - Loss:  2.844, Seconds: 793.47\n",
      "Valid Loss:  2.939, Seconds: 525.55\n",
      "New Record!\n",
      "Epoch   8/50 Batch  140/253 - Loss:  2.851, Seconds: 677.06\n",
      "Epoch   8/50 Batch  160/253 - Loss:  2.839, Seconds: 698.17\n",
      "Epoch   8/50 Batch  180/253 - Loss:  2.832, Seconds: 746.27\n",
      "Epoch   8/50 Batch  200/253 - Loss:  2.809, Seconds: 670.21\n",
      "Epoch   8/50 Batch  220/253 - Loss:  2.808, Seconds: 642.48\n",
      "Epoch   8/50 Batch  240/253 - Loss:  2.800, Seconds: 685.94\n",
      "Valid Loss:  2.981, Seconds: 505.30\n",
      "No Improvement.\n",
      "Epoch   9/50 Batch    0/253 - Loss:  1.857, Seconds: 639.56\n",
      "Epoch   9/50 Batch   20/253 - Loss:  2.804, Seconds: 818.19\n",
      "Epoch   9/50 Batch   40/253 - Loss:  2.754, Seconds: 696.53\n",
      "Epoch   9/50 Batch   60/253 - Loss:  2.720, Seconds: 655.53\n",
      "Epoch   9/50 Batch   80/253 - Loss:  2.809, Seconds: 685.38\n",
      "Epoch   9/50 Batch  100/253 - Loss:  2.747, Seconds: 711.95\n",
      "Epoch   9/50 Batch  120/253 - Loss:  2.761, Seconds: 689.81\n",
      "Valid Loss:  2.901, Seconds: 477.17\n",
      "New Record!\n",
      "Epoch   9/50 Batch  140/253 - Loss:  2.777, Seconds: 720.83\n",
      "Epoch   9/50 Batch  160/253 - Loss:  2.766, Seconds: 648.29\n",
      "Epoch   9/50 Batch  180/253 - Loss:  2.757, Seconds: 673.02\n",
      "Epoch   9/50 Batch  200/253 - Loss:  2.737, Seconds: 687.26\n",
      "Epoch   9/50 Batch  220/253 - Loss:  2.740, Seconds: 666.57\n",
      "Epoch   9/50 Batch  240/253 - Loss:  2.727, Seconds: 669.53\n",
      "Valid Loss:  2.943, Seconds: 424.93\n",
      "No Improvement.\n",
      "Epoch  10/50 Batch    0/253 - Loss:  1.815, Seconds: 632.71\n",
      "Epoch  10/50 Batch   20/253 - Loss:  2.734, Seconds: 794.84\n",
      "Epoch  10/50 Batch   40/253 - Loss:  2.687, Seconds: 710.44\n",
      "Epoch  10/50 Batch   60/253 - Loss:  2.656, Seconds: 774.53\n",
      "Epoch  10/50 Batch   80/253 - Loss:  2.740, Seconds: 705.56\n",
      "Epoch  10/50 Batch  100/253 - Loss:  2.682, Seconds: 646.57\n",
      "Epoch  10/50 Batch  120/253 - Loss:  2.696, Seconds: 616.03\n",
      "Valid Loss:  2.875, Seconds: 400.19\n",
      "New Record!\n",
      "Epoch  10/50 Batch  140/253 - Loss:  2.712, Seconds: 658.62\n",
      "Epoch  10/50 Batch  160/253 - Loss:  2.702, Seconds: 651.15\n",
      "Epoch  10/50 Batch  180/253 - Loss:  2.695, Seconds: 692.34\n",
      "Epoch  10/50 Batch  200/253 - Loss:  2.676, Seconds: 631.95\n",
      "Epoch  10/50 Batch  220/253 - Loss:  2.675, Seconds: 633.44\n",
      "Epoch  10/50 Batch  240/253 - Loss:  2.665, Seconds: 643.02\n",
      "Valid Loss:  2.928, Seconds: 383.40\n",
      "No Improvement.\n",
      "Epoch  11/50 Batch    0/253 - Loss:  1.772, Seconds: 572.31\n",
      "Epoch  11/50 Batch   20/253 - Loss:  2.669, Seconds: 670.90\n",
      "Epoch  11/50 Batch   40/253 - Loss:  2.622, Seconds: 639.27\n",
      "Epoch  11/50 Batch   60/253 - Loss:  2.591, Seconds: 588.31\n",
      "Epoch  11/50 Batch   80/253 - Loss:  2.672, Seconds: 647.43\n",
      "Epoch  11/50 Batch  100/253 - Loss:  2.616, Seconds: 616.93\n",
      "Epoch  11/50 Batch  120/253 - Loss:  2.635, Seconds: 593.55\n",
      "Valid Loss:  2.850, Seconds: 382.73\n",
      "New Record!\n",
      "Epoch  11/50 Batch  140/253 - Loss:  2.649, Seconds: 650.52\n",
      "Epoch  11/50 Batch  160/253 - Loss:  2.639, Seconds: 649.59\n",
      "Epoch  11/50 Batch  180/253 - Loss:  2.639, Seconds: 635.57\n",
      "Epoch  11/50 Batch  200/253 - Loss:  2.616, Seconds: 612.70\n",
      "Epoch  11/50 Batch  220/253 - Loss:  2.615, Seconds: 594.15\n",
      "Epoch  11/50 Batch  240/253 - Loss:  2.604, Seconds: 631.88\n",
      "Valid Loss:  2.904, Seconds: 380.50\n",
      "No Improvement.\n",
      "Epoch  12/50 Batch    0/253 - Loss:  1.735, Seconds: 577.28\n",
      "Epoch  12/50 Batch   20/253 - Loss:  2.615, Seconds: 664.29\n",
      "Epoch  12/50 Batch   40/253 - Loss:  2.573, Seconds: 628.47\n",
      "Epoch  12/50 Batch   60/253 - Loss:  2.533, Seconds: 612.19\n",
      "Epoch  12/50 Batch   80/253 - Loss:  2.618, Seconds: 645.88\n",
      "Epoch  12/50 Batch  100/253 - Loss:  2.563, Seconds: 628.74\n",
      "Epoch  12/50 Batch  120/253 - Loss:  2.580, Seconds: 597.84\n",
      "Valid Loss:  2.838, Seconds: 377.27\n",
      "New Record!\n",
      "Epoch  12/50 Batch  140/253 - Loss:  2.598, Seconds: 626.60\n",
      "Epoch  12/50 Batch  160/253 - Loss:  2.588, Seconds: 666.60\n",
      "Epoch  12/50 Batch  180/253 - Loss:  2.586, Seconds: 621.53\n",
      "Epoch  12/50 Batch  200/253 - Loss:  2.565, Seconds: 630.34\n",
      "Epoch  12/50 Batch  220/253 - Loss:  2.566, Seconds: 582.20\n",
      "Epoch  12/50 Batch  240/253 - Loss:  2.554, Seconds: 603.49\n",
      "Valid Loss:  2.888, Seconds: 379.83\n",
      "No Improvement.\n",
      "Epoch  13/50 Batch    0/253 - Loss:  1.703, Seconds: 567.31\n",
      "Epoch  13/50 Batch   20/253 - Loss:  2.561, Seconds: 733.87\n",
      "Epoch  13/50 Batch   40/253 - Loss:  2.520, Seconds: 659.48\n",
      "Epoch  13/50 Batch   60/253 - Loss:  2.488, Seconds: 621.57\n",
      "Epoch  13/50 Batch   80/253 - Loss:  2.574, Seconds: 653.04\n",
      "Epoch  13/50 Batch  100/253 - Loss:  2.512, Seconds: 635.32\n",
      "Epoch  13/50 Batch  120/253 - Loss:  2.535, Seconds: 602.31\n",
      "Valid Loss:  2.831, Seconds: 377.81\n",
      "New Record!\n",
      "Epoch  13/50 Batch  140/253 - Loss:  2.553, Seconds: 642.06\n",
      "Epoch  13/50 Batch  160/253 - Loss:  2.545, Seconds: 743.52\n",
      "Epoch  13/50 Batch  180/253 - Loss:  2.542, Seconds: 687.73\n",
      "Epoch  13/50 Batch  200/253 - Loss:  2.518, Seconds: 706.94\n",
      "Epoch  13/50 Batch  220/253 - Loss:  2.526, Seconds: 638.69\n",
      "Epoch  13/50 Batch  240/253 - Loss:  2.509, Seconds: 664.35\n",
      "Valid Loss:  2.867, Seconds: 378.13\n",
      "No Improvement.\n",
      "Epoch  14/50 Batch    0/253 - Loss:  1.672, Seconds: 558.27\n",
      "Epoch  14/50 Batch   20/253 - Loss:  2.514, Seconds: 732.83\n",
      "Epoch  14/50 Batch   40/253 - Loss:  2.481, Seconds: 701.73\n",
      "Epoch  14/50 Batch   60/253 - Loss:  2.444, Seconds: 679.89\n",
      "Epoch  14/50 Batch   80/253 - Loss:  2.532, Seconds: 764.76\n",
      "Epoch  14/50 Batch  100/253 - Loss:  2.470, Seconds: 703.62\n",
      "Epoch  14/50 Batch  120/253 - Loss:  2.491, Seconds: 670.86\n",
      "Valid Loss:  2.827, Seconds: 376.97\n",
      "New Record!\n",
      "Epoch  14/50 Batch  140/253 - Loss:  2.508, Seconds: 714.92\n",
      "Epoch  14/50 Batch  160/253 - Loss:  2.503, Seconds: 724.51\n",
      "Epoch  14/50 Batch  180/253 - Loss:  2.497, Seconds: 639.00\n",
      "Epoch  14/50 Batch  200/253 - Loss:  2.481, Seconds: 665.89\n",
      "Epoch  14/50 Batch  220/253 - Loss:  2.483, Seconds: 651.79\n",
      "Epoch  14/50 Batch  240/253 - Loss:  2.469, Seconds: 643.08\n",
      "Valid Loss:  2.865, Seconds: 377.13\n",
      "No Improvement.\n",
      "Epoch  15/50 Batch    0/253 - Loss:  1.646, Seconds: 638.38\n",
      "Epoch  15/50 Batch   20/253 - Loss:  2.474, Seconds: 721.70\n",
      "Epoch  15/50 Batch   40/253 - Loss:  2.441, Seconds: 714.50\n",
      "Epoch  15/50 Batch   60/253 - Loss:  2.408, Seconds: 649.28\n",
      "Epoch  15/50 Batch   80/253 - Loss:  2.493, Seconds: 821.89\n",
      "Epoch  15/50 Batch  100/253 - Loss:  2.433, Seconds: 731.39\n",
      "Epoch  15/50 Batch  120/253 - Loss:  2.458, Seconds: 682.96\n",
      "Valid Loss:  2.824, Seconds: 369.21\n",
      "New Record!\n",
      "Epoch  15/50 Batch  140/253 - Loss:  2.469, Seconds: 686.96\n",
      "Epoch  15/50 Batch  160/253 - Loss:  2.462, Seconds: 685.22\n",
      "Epoch  15/50 Batch  180/253 - Loss:  2.466, Seconds: 670.84\n",
      "Epoch  15/50 Batch  200/253 - Loss:  2.443, Seconds: 683.67\n",
      "Epoch  15/50 Batch  220/253 - Loss:  2.440, Seconds: 646.16\n",
      "Epoch  15/50 Batch  240/253 - Loss:  2.433, Seconds: 692.41\n",
      "Valid Loss:  2.861, Seconds: 372.13\n",
      "No Improvement.\n",
      "Epoch  16/50 Batch    0/253 - Loss:  1.622, Seconds: 636.33\n",
      "Epoch  16/50 Batch   20/253 - Loss:  2.438, Seconds: 694.87\n",
      "Epoch  16/50 Batch   40/253 - Loss:  2.406, Seconds: 693.31\n",
      "Epoch  16/50 Batch   60/253 - Loss:  2.373, Seconds: 767.54\n",
      "Epoch  16/50 Batch   80/253 - Loss:  2.464, Seconds: 837.77\n",
      "Epoch  16/50 Batch  100/253 - Loss:  2.400, Seconds: 776.42\n",
      "Epoch  16/50 Batch  120/253 - Loss:  2.426, Seconds: 760.49\n",
      "Valid Loss:  2.817, Seconds: 441.51\n",
      "New Record!\n",
      "Epoch  16/50 Batch  140/253 - Loss:  2.439, Seconds: 823.47\n",
      "Epoch  16/50 Batch  160/253 - Loss:  2.428, Seconds: 825.56\n",
      "Epoch  16/50 Batch  180/253 - Loss:  2.431, Seconds: 822.52\n",
      "Epoch  16/50 Batch  200/253 - Loss:  2.410, Seconds: 904.12\n",
      "Epoch  16/50 Batch  220/253 - Loss:  2.412, Seconds: 819.09\n",
      "Epoch  16/50 Batch  240/253 - Loss:  2.401, Seconds: 759.76\n",
      "Valid Loss:  2.859, Seconds: 430.86\n",
      "No Improvement.\n",
      "Epoch  17/50 Batch    0/253 - Loss:  1.602, Seconds: 759.87\n",
      "Epoch  17/50 Batch   20/253 - Loss:  2.405, Seconds: 1042.42\n",
      "Epoch  17/50 Batch   40/253 - Loss:  2.370, Seconds: 789.28\n",
      "Epoch  17/50 Batch   60/253 - Loss:  2.343, Seconds: 766.48\n",
      "Epoch  17/50 Batch   80/253 - Loss:  2.430, Seconds: 979.68\n",
      "Epoch  17/50 Batch  100/253 - Loss:  2.372, Seconds: 875.60\n",
      "Epoch  17/50 Batch  120/253 - Loss:  2.395, Seconds: 794.50\n",
      "Valid Loss:  2.820, Seconds: 447.26\n",
      "No Improvement.\n",
      "Epoch  17/50 Batch  140/253 - Loss:  2.412, Seconds: 974.45\n",
      "Epoch  17/50 Batch  160/253 - Loss:  2.401, Seconds: 965.60\n",
      "Epoch  17/50 Batch  180/253 - Loss:  2.404, Seconds: 846.68\n",
      "Epoch  17/50 Batch  200/253 - Loss:  2.382, Seconds: 767.48\n",
      "Epoch  17/50 Batch  220/253 - Loss:  2.383, Seconds: 816.88\n",
      "Epoch  17/50 Batch  240/253 - Loss:  2.367, Seconds: 782.68\n",
      "Valid Loss:  2.865, Seconds: 456.20\n",
      "No Improvement.\n",
      "Epoch  18/50 Batch    0/253 - Loss:  1.577, Seconds: 862.89\n",
      "Epoch  18/50 Batch   20/253 - Loss:  2.375, Seconds: 936.87\n",
      "Epoch  18/50 Batch   40/253 - Loss:  2.345, Seconds: 779.70\n",
      "Epoch  18/50 Batch   60/253 - Loss:  2.312, Seconds: 866.03\n",
      "Epoch  18/50 Batch   80/253 - Loss:  2.399, Seconds: 992.80\n",
      "Epoch  18/50 Batch  100/253 - Loss:  2.341, Seconds: 912.05\n",
      "Epoch  18/50 Batch  120/253 - Loss:  2.364, Seconds: 1217.18\n",
      "Valid Loss:  2.817, Seconds: 690.11\n",
      "New Record!\n",
      "Epoch  18/50 Batch  140/253 - Loss:  2.383, Seconds: 1142.64\n",
      "Epoch  18/50 Batch  160/253 - Loss:  2.372, Seconds: 1321.16\n",
      "Epoch  18/50 Batch  180/253 - Loss:  2.377, Seconds: 812.36\n",
      "Epoch  18/50 Batch  200/253 - Loss:  2.352, Seconds: 1170.80\n",
      "Epoch  18/50 Batch  220/253 - Loss:  2.355, Seconds: 1485.47\n",
      "Epoch  18/50 Batch  240/253 - Loss:  2.337, Seconds: 1268.44\n",
      "Valid Loss:  2.859, Seconds: 496.87\n",
      "No Improvement.\n",
      "Epoch  19/50 Batch    0/253 - Loss:  1.563, Seconds: 1146.02\n",
      "Epoch  19/50 Batch   20/253 - Loss:  2.347, Seconds: 1045.75\n",
      "Epoch  19/50 Batch   40/253 - Loss:  2.318, Seconds: 1037.51\n",
      "Epoch  19/50 Batch   60/253 - Loss:  2.287, Seconds: 998.97\n",
      "Epoch  19/50 Batch   80/253 - Loss:  2.368, Seconds: 1170.27\n",
      "Epoch  19/50 Batch  100/253 - Loss:  2.318, Seconds: 1214.10\n",
      "Epoch  19/50 Batch  120/253 - Loss:  2.337, Seconds: 1053.31\n",
      "Valid Loss:  2.822, Seconds: 468.68\n",
      "No Improvement.\n",
      "Epoch  19/50 Batch  140/253 - Loss:  2.352, Seconds: 1066.85\n",
      "Epoch  19/50 Batch  160/253 - Loss:  2.345, Seconds: 1236.45\n",
      "Epoch  19/50 Batch  180/253 - Loss:  2.349, Seconds: 1284.28\n",
      "Epoch  19/50 Batch  200/253 - Loss:  2.327, Seconds: 1066.12\n",
      "Epoch  19/50 Batch  220/253 - Loss:  2.324, Seconds: 1195.94\n",
      "Epoch  19/50 Batch  240/253 - Loss:  2.307, Seconds: 1211.90\n",
      "Valid Loss:  2.871, Seconds: 471.52\n",
      "No Improvement.\n",
      "Epoch  20/50 Batch    0/253 - Loss:  1.545, Seconds: 1168.90\n",
      "Epoch  20/50 Batch   20/253 - Loss:  2.320, Seconds: 1146.65\n",
      "Epoch  20/50 Batch   40/253 - Loss:  2.296, Seconds: 1121.44\n",
      "Epoch  20/50 Batch   60/253 - Loss:  2.265, Seconds: 1192.79\n",
      "Epoch  20/50 Batch   80/253 - Loss:  2.350, Seconds: 1489.93\n",
      "Epoch  20/50 Batch  100/253 - Loss:  2.281, Seconds: 1276.95\n",
      "Epoch  20/50 Batch  120/253 - Loss:  2.308, Seconds: 1186.60\n",
      "Valid Loss:  2.823, Seconds: 453.69\n",
      "No Improvement.\n",
      "Epoch  20/50 Batch  140/253 - Loss:  2.329, Seconds: 1309.86\n",
      "Epoch  20/50 Batch  160/253 - Loss:  2.322, Seconds: 1472.13\n",
      "Epoch  20/50 Batch  180/253 - Loss:  2.321, Seconds: 1300.82\n",
      "Epoch  20/50 Batch  200/253 - Loss:  2.304, Seconds: 1282.79\n",
      "Epoch  20/50 Batch  220/253 - Loss:  2.300, Seconds: 1140.20\n",
      "Epoch  20/50 Batch  240/253 - Loss:  2.284, Seconds: 1185.98\n",
      "Valid Loss:  2.874, Seconds: 464.63\n",
      "No Improvement.\n",
      "Stopping Training.\n"
     ]
    }
   ],
   "source": [
    "for epoch_i in range(1, epochs+1):\n",
    "    for batch_i, (questions_batch, answers_batch) in enumerate(\n",
    "            batch_data(train_questions, train_answers, batch_size)):\n",
    "        start_time = time.time()\n",
    "        _, loss = sess.run(\n",
    "            [train_op, cost],\n",
    "            {input_data: questions_batch, targets: answers_batch,  lr: learning_rate, \n",
    "             sequence_length: answers_batch.shape[1], keep_prob: keep_probability})\n",
    "\n",
    "        total_train_loss += loss\n",
    "        end_time = time.time()\n",
    "        batch_time = end_time - start_time\n",
    "\n",
    "        if batch_i % display_step == 0:\n",
    "            print('Epoch {:>3}/{} Batch {:>4}/{} - Loss: {:>6.3f}, Seconds: {:>4.2f}'\n",
    "                  .format(epoch_i, epochs, batch_i, \n",
    "                          len(train_questions) // batch_size, total_train_loss / display_step, \n",
    "                          batch_time*display_step))\n",
    "            total_train_loss = 0\n",
    "\n",
    "        if batch_i % validation_check == 0 and batch_i > 0:\n",
    "            total_valid_loss = 0\n",
    "            start_time = time.time()\n",
    "            for batch_ii, (questions_batch, answers_batch) in enumerate(batch_data(valid_questions, valid_answers, batch_size)):\n",
    "                valid_loss = sess.run(\n",
    "                cost, {input_data: questions_batch, targets: answers_batch, lr: learning_rate, \n",
    "                       sequence_length: answers_batch.shape[1], keep_prob: 1})\n",
    "                total_valid_loss += valid_loss\n",
    "            end_time = time.time()\n",
    "            batch_time = end_time - start_time\n",
    "            avg_valid_loss = total_valid_loss / (len(valid_questions) / batch_size)\n",
    "            print('Valid Loss: {:>6.3f}, Seconds: {:>5.2f}'.format(avg_valid_loss, batch_time))\n",
    "            \n",
    "            # Reduce learning rate with a minimum value threshold\n",
    "            learning_rate *= learning_rate_decay\n",
    "            if learning_rate < min_learning_rate:\n",
    "                learning_rate = min_learning_rate\n",
    "\n",
    "            summary_valid_loss.append(avg_valid_loss)\n",
    "            if avg_valid_loss <= min(summary_valid_loss):\n",
    "                print('New Record!') \n",
    "                stop_early = 0\n",
    "                saver = tf.train.Saver() \n",
    "                saver.save(sess, checkpoint)\n",
    "\n",
    "            else:\n",
    "                print(\"No Improvement.\")\n",
    "                stop_early += 1\n",
    "                if stop_early == stop:\n",
    "                    break\n",
    "    \n",
    "    if stop_early == stop:\n",
    "        print(\"Stopping Training.\")\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def question_to_seq(question, vocab_to_int):\n",
    "    \"\"\"Creating the question to be taken as input by the model\"\"\"\n",
    "    question = clean_text(question)\n",
    "    return [vocab_to_int.get(word, vocab_to_int['<UNK>']) for word in question.split()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "what exactly does adjustable life insurance mean\n",
      "Question\n",
      "  Word Ids:      [17288, 16123, 9831, 13347, 1694, 11205, 7655]\n",
      "  Input Words: ['what', 'exactly', 'does', 'adjustable', 'life', 'insurance', 'mean']\n",
      "\n",
      "\n",
      "what exactly does adjustable life insurance mean\n",
      "\n",
      "Answer\n",
      "  Word Ids:      [10130, 10344, 13123, 2313, 1133, 1694, 11205, 6968, 966, 10130, 3030, 2313, 5964, 10561, 10130, 9158, 17702, 13344, 13278, 10130, 7457, 14167, 17931, 14479, 10130, 6968, 9158, 8521, 10130, 9158, 17702, 12230, 10130, 6968, 8679, 1688, 10130, 7457, 14167, 17931, 9472, 10130, 9158, 12230, 10130, 6968, 8679, 1688, 10130, 7457, 14167, 17931, 18293, 10130, 16405, 16640, 6396, 3613, 2313, 10130, 6968, 10130, 6968, 8679, 1688, 10130, 7457, 14167, 17931, 18293, 10130, 16405, 16640, 6396, 3613, 10628, 13040, 10130, 6968]\n",
      "  Response Words: ['the', 'face', 'value', 'of', 'a', 'life', 'insurance', 'policy', 'is', 'the', 'amount', 'of', 'time', 'that', 'the', 'insured', 'person', 'passes', 'with', 'the', 'death', 'benefit', 'proceeds', 'from', 'the', 'policy', 'insured', 'if', 'the', 'insured', 'person', 'dies', 'the', 'policy', 'will', 'pay', 'the', 'death', 'benefit', 'proceeds', 'whenever', 'the', 'insured', 'dies', 'the', 'policy', 'will', 'pay', 'the', 'death', 'benefit', 'proceeds', 'within', 'the', 'two', 'year', 'contestability', 'period', 'of', 'the', 'policy', 'the', 'policy', 'will', 'pay', 'the', 'death', 'benefit', 'proceeds', 'within', 'the', 'two', 'year', 'contestability', 'period', 'specified', 'in', 'the', 'policy']\n",
      "\n",
      "\n",
      "the face value of a life insurance policy is the amount of time that the insured person passes with the death benefit proceeds from the policy insured if the insured person dies the policy will pay the death benefit proceeds whenever the insured dies the policy will pay the death benefit proceeds within the two year contestability period of the policy the policy will pay the death benefit proceeds within the two year contestability period specified in the policy\n"
     ]
    }
   ],
   "source": [
    "# Selecting a random question from the full lot\n",
    "random = np.random.choice(len(short_questions))\n",
    "input_question = short_questions[random]\n",
    "print(input_question)\n",
    "\n",
    "# Transforming the selected question in the desired format of IDs and Words\n",
    "input_question = question_to_seq(input_question, questions_vocab_to_int)\n",
    "\n",
    "# Applying Padding to the question to reach the max_line_length\n",
    "input_question = input_question + [questions_vocab_to_int[\"<PAD>\"]] * (max_line_length - len(input_question))\n",
    "\n",
    "# Correcting the shape of input_data, by adding the empty questions \n",
    "batch_shell = np.zeros((batch_size, max_line_length))\n",
    "\n",
    "# Setting the input question as the first question\n",
    "batch_shell[0] = input_question    \n",
    "    \n",
    "# Passing  input question to the model\n",
    "answer_logits = sess.run(inference_logits, {input_data: batch_shell, \n",
    "                                            keep_prob: 1.0})[0]\n",
    "\n",
    "# Removing padding from Question and Answer both\n",
    "pad_q = questions_vocab_to_int[\"<PAD>\"]\n",
    "pad_a = answers_vocab_to_int[\"<PAD>\"]\n",
    "\n",
    "# Printing the final Answer output by the model \n",
    "print('Question')\n",
    "print('  Word Ids:      {}'.format([i for i in input_question if i != pad_q]))\n",
    "print('  Input Words: {}'.format([questions_int_to_vocab[i] for i in input_question if i != pad_q]))\n",
    "print('\\n')\n",
    "print(' '.join(([questions_int_to_vocab[i] for i in input_question if i != pad_q])))\n",
    "\n",
    "print('\\nAnswer')\n",
    "print('  Word Ids:      {}'.format([i for i in np.argmax(answer_logits, 1) if i != pad_a]))\n",
    "print('  Response Words: {}'.format([answers_int_to_vocab[i] for i in np.argmax(answer_logits, 1) if i != pad_a]))\n",
    "print('\\n')\n",
    "print(' '.join(([answers_int_to_vocab[i] for i in np.argmax(answer_logits, 1) if i != pad_a])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
